{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-ef312f74e334>:4: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From d:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From d:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "(100, 784)\n",
      "(100, 10)\n",
      "mnist.train.num_examples=55000\n",
      "n_batch=550\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "#data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "data_dir = \"data\"\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "print(batch_xs.shape)\n",
    "print(batch_ys.shape)\n",
    "#每个批次大小\n",
    "#batch_size = 100\n",
    "#在PC 上调低bach_size大小\n",
    "batch_size = 100\n",
    "#一共多少个批次\n",
    "#n_batch = 3000 // batch_size\n",
    "n_batch = mnist.train.num_examples // batch_size\n",
    "print(\"mnist.train.num_examples=%d\" % mnist.train.num_examples)\n",
    "print(\"n_batch=%d\" % n_batch)\n",
    "#初始化W \n",
    "def W_variable(shape,stddev=0.1):\n",
    "    init = tf.truncated_normal(shape=shape,mean=0,stddev=0.1)\n",
    "    return tf.Variable(init)\n",
    "\n",
    "#初始化Bias\n",
    "def B_variable(shape):\n",
    "    init = tf.constant(value=0.1, shape=shape)\n",
    "    return tf.Variable(init)\n",
    "    \n",
    "#卷基层\n",
    "def conv2d(x,W):\n",
    "    return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')\n",
    "\n",
    "#最大池化层\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')\n",
    "\n",
    "#平均池化层\n",
    "def mean_pool_2x2(x):\n",
    "    return tf.nn.avg_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "#init_learning_rate = tf.placeholder(tf.float32)\n",
    "x_image = tf.reshape(x,[-1,28,28,1])\n",
    "\n",
    "#W = tf.Variable(tf.zeros([784, 10]))\n",
    "#第一个卷基层W\n",
    "W1 = W_variable([5,5,1,32])\n",
    "#b = tf.Variable(tf.zeros([10]))\n",
    "#第一个卷积层的Bias\n",
    "b1 = B_variable([32])\n",
    "\n",
    "#进行第一层卷积运算,应用relu激活函数\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image,W1) + b1)\n",
    "#第一层池化\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "#平均池化\n",
    "#h_pool1 = mean_pools_2x2(h_conv1)\n",
    "                       \n",
    "W2 = W_variable([5,5,32,64])\n",
    "b2 = B_variable([64])\n",
    "                       \n",
    "#第二层卷积\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1,W2) + b2)\n",
    "#第二层池化\n",
    "h_pool2 =  max_pool_2x2(h_conv2)\n",
    "\n",
    "#第一次池化后变为14*14\n",
    "#第二次池化为7*7，得到64个平面\n",
    "# = tf.matmul(x, W) + b\n",
    "#初始化第一个全连接层\n",
    "W_fc1 = W_variable([7*7*64,1024])\n",
    "b_fc1 =  B_variable([1024])\n",
    "#讲第二池化层扁平化\n",
    "h_pool2_flat = tf.reshape(h_pool2,[-1,7*7*64])\n",
    "#第一个全连接层输出\n",
    "h_fc1= tf.nn.relu(tf.matmul(h_pool2_flat,W_fc1) + b_fc1)\n",
    "\n",
    "#神经元输出概率\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1,keep_prob)\n",
    "\n",
    "#初始化第二个全连接层\n",
    "W_fc2 = W_variable([1024,10])\n",
    "b_fc2 = B_variable([10])\n",
    "\n",
    "prediction = tf.nn.softmax(tf.matmul(h_fc1_drop,W_fc2) + b_fc2)\n",
    "\n",
    "# epoch_steps = tf.to_int64(tf.div(60000, tf.shape(x)[0]))\n",
    "# global_step = tf.train.get_or_create_global_step()\n",
    "# current_epoch = global_step//epoch_steps\n",
    "# decay_times = current_epoch \n",
    "# current_learning_rate = tf.multiply(init_learning_rate, \n",
    "#                                     tf.pow(0.575, tf.to_float(decay_times)))\n",
    "current_learning_rate =1e-4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-121d9af177f1>:11: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "#交叉熵代价函数\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=prediction))\n",
    "#关于l2 正则，前面应用了dropout()函数减少过拟合，不应再增加L2正则减小过拟合了。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "#此处使用adam优化器\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(current_learning_rate).minimize(cross_entropy)\n",
    "\n",
    "# Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(prediction, axis=1), tf.argmax(y_, axis=1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "testing accuracy= 99 0.63\n",
      "testing accuracy= 199 0.84\n",
      "testing accuracy= 299 0.76\n",
      "testing accuracy= 399 0.92\n",
      "testing accuracy= 499 0.91\n",
      "testing accuracy= 599 0.95\n",
      "testing accuracy= 699 0.97\n",
      "testing accuracy= 799 0.97\n",
      "testing accuracy= 899 0.95\n",
      "testing accuracy= 999 0.94\n",
      "testing accuracy= 1099 0.95\n",
      "testing accuracy= 1199 0.99\n",
      "testing accuracy= 1299 0.99\n",
      "testing accuracy= 1399 0.98\n",
      "testing accuracy= 1499 0.96\n",
      "testing accuracy= 1599 1.0\n",
      "testing accuracy= 1699 0.99\n",
      "testing accuracy= 1799 0.97\n",
      "testing accuracy= 1899 0.96\n",
      "testing accuracy= 1999 0.99\n",
      "testing accuracy= 2099 0.99\n",
      "testing accuracy= 2199 0.97\n",
      "testing accuracy= 2299 0.98\n",
      "testing accuracy= 2399 0.98\n",
      "testing accuracy= 2499 0.96\n",
      "testing accuracy= 2599 0.99\n",
      "testing accuracy= 2699 1.0\n",
      "testing accuracy= 2799 0.99\n",
      "testing accuracy= 2899 0.99\n",
      "testing accuracy= 2999 0.99\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "with tf.Session() as sess:\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    #for epoch in range(20):\n",
    "    for batch in range(3000):\n",
    "        batch_xs,batch_ys = mnist.train.next_batch(batch_size)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.7})\n",
    "        if (batch +1) % 100 == 0:\n",
    "            print(\"testing accuracy=\" ,batch,\n",
    "                  sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 1.0}))\n",
    "        #验证我们模型在测试数据上的准确率\n",
    "    acc = sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels, keep_prob:1.0})\n",
    "    print(\"finally testing accuracy=\" + str(acc))\n",
    "       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 卷积\n",
    "- 池化\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 卷积kernel size\n",
    "  - 卷积kernel 数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
